#!/bin/bash -x

work_top_dir="$HOME/workspace"
source /etc/os-release

if [ ! -d ${work_top_dir} ]; then
  mkdir -p ${work_top_dir}
fi

usage () {
  set +x
  echo -e "\033[31m
Usage: $0 [-h] [-r <k8s-node-role>] [-n <k8s-node-name>]

       -h,--help Display this help message
       -r,--role Specify the k8s node role, default is control-plane
                 control-plane: means k8s master node, cannot deploy CoCo on this node
                 control-plane+worker: means k8s master node, can deploy CoCo on this node
                 worker: means k8s worker node (can deploy CoCo on this node), this node should join the k8s master node
       -n,--name Set the name of the k8s node. If not set, the default name is value of $HOSTNAME
\033[0m"
  set -x
}

k8s_node_role="control-plane"
k8s_node_name="$HOSTNAME"
params=$(getopt -o hr:n: --long help,role:,name: -- "$@")
if [ $? -ne 0 ]; then
  echo "[Error] cannot parse parameters"
  exit 1
fi

eval set -- "$params"

while true; do
  case "$1" in
  -h|--help)
    usage
    exit 0
    ;;
  -r|--role)
    k8s_node_role="$2"
    shift 2
    ;;
  -n|--name)
    k8s_node_name="$2"
    shift 2
    ;;
  --)
    shift
    break
    ;;
  *)
    usage
    exit 1
    ;;
  esac
done

check_set_cgroup_version () {
  echo "Check & set cgroup V1"
  cgroup_v1_cmdline="systemd.unified_cgroup_hierarchy=0"
  cgroup_fs_type=$(stat -fc %T /sys/fs/cgroup/)

  if [ "$cgroup_fs_type" != "tmpfs" ];then
    if [ $(grep -c "${cgroup_v1_cmdline}" /proc/cmdline) -eq 0 ]; then
      echo -e "\033[31;1m[Error]\033[0m We must use cgroup V1 for the combination: container 1.7.x + k8s v1.28.0"
      sudo sed -i "s/^\(GRUB_CMDLINE_LINUX=\".*\)\"$/\1 ${cgroup_v1_cmdline}\"/g" /etc/default/grub
      sudo grub2-mkconfig -o /boot/efi/EFI/$ID/grub.cfg
      read -p "You must reboot the host to using the cgroup V1, Press [y/Y/n/N]" do_reboot
      case "$do_reboot" in
      "y" | "Y")
        sudo reboot
        ;;
      "n" | "N" | *)
        echo -e "\033[31;7mYou can not move forward !!! Please reboot\033[0m"
        exit 1
        ;;
      esac
    fi
  fi
}

check_set_cgroup_version

# reset + cleanup
do_cleanup () {
  echo "Do cleanup"
  command -v kubectl > /dev/null
  if [ $? -eq 0 ]; then
  kubectl delete -k $(pwd)/trustee/kbs/config/kubernetes/overlays || true
  kubectl delete -k $(pwd)/operator/config/samples/ccruntime/default/ || true
  kubectl delete -k $(pwd)/operator/config/release/ || true
  fi

  command -v kubeadm > /dev/null
  if [ $? -eq 0 ]; then
  sudo kubeadm reset
  fi
  sudo rm -rf /sys/fs/cgroup/kubepods.slice || true

  command -v ctr > /dev/null
  if [ $? -eq 0 ]; then
  sudo ctr -n k8s.io image ls | awk '{print $1}' | xargs -n1 sudo ctr -n k8s.io image rm
  fi

  sudo systemctl stop docker || true
  sudo systemctl stop containerd || true
  sudo pkill -f containerd || true
  # cleanup metadatas, if not, the nydus may work improperly
  sudo rm -rf /tmp/nydus-snapshotter || true # just for local
  sudo rm -rf /opt/confidential-containers || true
  sudo rm -rf /opt/containerd/ || true
  sudo rm -rf /var/run/containerd-nydus || true
  sudo rm -rf /var/run/containerd || true
  sudo rm -rf /var/lib/containerd-nydus || true
  sudo rm -rf /var/lib/containerd || true
  sync
}

do_cleanup

setup_system () {
  echo "Setup system"
  # disable firewalld
  sudo systemctl disable firewalld --now
  # disable selinux
  sudo setenforce 0
  sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
  # inhibit swap
  # 临时禁用
  sudo swapoff -a
  # 永久禁用，编辑 /etc/fstab 文件，注释掉包含 swap 的行
  sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

  # ========================================================================
  # kernel params
  # 加载必要的内核模块
  sudo modprobe overlay
  sudo modprobe br_netfilter

  # 创建并编辑系统参数文件
  cat <<EOF | sudo tee /etc/sysctl.d/kubernetes.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

  # 应用系统参数
  sudo sysctl --system
}

setup_system

install_basic_deps () {
  echo "Install basic dependencies"

  if [ "$ID" = "anolis" ]; then
    if [ ! -e /etc/yum.repos.d/AnolisOS-DDE.repo ]; then
      sudo tee /etc/yum.repos.d/AnolisOS-DDE.repo <<EOF
[DDE]
name=AnolisOS-\$releasever - DDE
baseurl=http://mirrors.openanolis.cn/anolis/\$releasever/DDE/\$basearch/os
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-ANOLIS
gpgcheck=1
EOF
    fi

    if [ ! -e /etc/yum.repos.d/AnolisOS-Devel.repo ]; then
      sudo tee /etc/yum.repos.d/AnolisOS-Devel.repo <<EOF
[Devel]
name=AnolisOS-\$releasever - Devel
baseurl=http://mirrors.openanolis.cn/anolis/\$releasever/Devel/\$basearch/os
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-ANOLIS
gpgcheck=1
EOF
    fi

    if [ ! -e /etc/yum.repos.d/AnolisOS-Nightly.repo ]; then
      sudo tee /etc/yum.repos.d/AnolisOS-Nightly.repo <<EOF
[Nightly]
name=AnolisOS-\$releasever - Nightly
baseurl=http://mirrors.openanolis.cn/anolis/\$releasever/Nightly/os/\$basearch/os
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-ANOLIS
gpgcheck=0
EOF
    fi
  else
    if [ ! -e /etc/yum.repos.d/kubernetes.repo ]; then
    sudo tee /etc/yum.repos.d/kubernetes.repo  <<EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
    fi
  fi
  sudo yum -y makecache

  sudo yum install -y curl wget
  sudo yum install -y yum-utils \
    device-mapper-persistent-data lvm2
}

install_basic_deps

install_docker_containerd () {
  echo "Install docker & containerd"
  # Remove docker & containerd installed from yum repos, we'll install official
  # release
  sudo yum remove -y docker \
    docker-client \
    docker-client-latest \
    docker-common \
    docker-latest \
    docker-latest-logrotate \
    docker-logrotate \
    docker-engine

  # Install fuse, containerd & nydus-snapshotter need fuse
  sudo yum install -y fuse

  # Download and install docker's official release, this release contains
  # containerd
  sudo mkdir -p /usr/local/bin
  if [ ! -e ${work_top_dir}/docker-28.0.2.tgz ]; then
    wget -O ${work_top_dir}/docker-28.0.2.tgz \
      https://download.docker.com/linux/static/stable/x86_64/docker-28.0.2.tgz
  fi
  sudo tar -zxvf ${work_top_dir}/docker-28.0.2.tgz \
    -C /usr/local/bin --strip-components=1

  # Create docker & containerd service
  sudo mkdir -p /usr/local/lib/systemd/system/
  sudo tee /usr/local/lib/systemd/system/containerd.service <<EOF
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target dbus.service

[Service]
#uncomment to enable the experimental sbservice (sandboxed) version of containerd/cri integration
#Environment="ENABLE_CRI_SANDBOXES=sandboxed"
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd

Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF
  sudo tee /usr/local/lib/systemd/system/docker.socket <<EOF
[Unit]
Description=Docker Socket for the API

[Socket]
# If /var/run is not implemented as a symlink to /run, you may need to
# specify ListenStream=/var/run/docker.sock instead.
ListenStream=/run/docker.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF
  sudo tee /usr/local/lib/systemd/system/docker.service <<EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target nss-lookup.target docker.socket firewalld.service containerd.service time-set.target
Wants=network-online.target containerd.service
Requires=docker.socket

[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/local/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP \$MAINPID
TimeoutStartSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity

# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes

# kill only the docker process, not all processes in the cgroup
KillMode=process
OOMScoreAdjust=-500

[Install]
WantedBy=multi-user.target
EOF

  sudo groupadd docker || true
  sudo usermod -aG docker $USER
  sudo systemctl daemon-reload && \
  sudo systemctl enable containerd && \
  sudo systemctl enable docker
  sudo systemctl restart containerd && \
  sudo systemctl restart docker

  # Install docker buildx
  docker_buildx_url="https://github.com/docker/buildx/releases/download/v0.20.0/buildx-v0.20.0.linux-amd64"
  docker_buildx_cache="${work_top_dir}/buildx-v0.20.0.linux-amd64"
  docker_buildx_path="/usr/local/lib/docker/cli-plugins/docker-buildx"
  if [ ! -e ${docker_buildx_cache} ]; then
    curl -SL ${docker_buildx_url} -o ${docker_buildx_cache}
  fi
  if [ ! -e ${docker_buildx_path} ]; then
    sudo mkdir -p $(dirname ${docker_buildx_path}) || true
    sudo cp ${docker_buildx_cache} ${docker_buildx_path}
    sudo chmod +x ${docker_buildx_path}
  fi

  docker buildx version
  sudo docker buildx version

  # Setup registry mirrors for docker & containerd
  registry_mirror_list="\"https:\/\/docker.m.daocloud.io\",\"https:\/\/noohub.ru\",\"https:\/\/huecker.io\",\"https:\/\/dockerhub.timeweb.cloud\",\"https:\/\/registry.cn-hangzhou.aliyuncs.com\",\"https:\/\/mirror.ccs.tencentyun.com\",\"https:\/\/docker.mirrors.ustc.edu.cn\",\"https:\/\/hub-mirror.c.163.com\""

  if [ -e /etc/docker/daemon.json ] && [ $(wc -l /etc/docker/daemon.json | awk '{print $1}') -gt 2 ]; then
    if [ $(grep -c "\"registry-mirrors\":" /etc/docker/daemon.json) -eq 0 ]; then
      sudo sed -i "1a\  \"registry-mirrors\": [${registry_mirror_list}]," \
        /etc/docker/daemon.json
    fi
  else
    sudo mkdir -p /etc/docker || true
    cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "registry-mirrors": [$(echo ${registry_mirror_list} | sed "s!\\\/!\/!g")]
}
EOF
  fi
  sudo mkdir -p /etc/containerd/
  sudo containerd config default | sudo tee /etc/containerd/config.toml
  sudo sed -i 's!\(sandbox_image = \).*pause:.*$!\1"registry.aliyuncs.com/google_containers/pause:3.9"!' \
    /etc/containerd/config.toml
  #sudo sed -i 's!\(systemd_cgroup = \).*$!\1true!' /etc/containerd/config.toml # just for debug use, don't uncomment this line
  #sudo sed -i 's!\(SystemdCgroup = \).*$!\1true!' /etc/containerd/config.toml # just for debug use, don't uncomment this line
  containerd_registry_mirror_marker="\[plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors\]"
  containerd_registry_mirror_docker_marker="\[plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"\]"
  containerd_registry_mirror_gcr_marker="\[plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"k8s.gcr.io\"\]"
  if [ $(sudo grep -c "${containerd_registry_mirror_marker}" /etc/containerd/config.toml) -ne 0 ]; then
    if [ $(sudo grep -c "${containerd_registry_mirror_docker_marker}" /etc/containerd/config.toml) -eq 0 ]; then
      sudo sed -i "/${containerd_registry_mirror_marker}/a\\
        ${containerd_registry_mirror_docker_marker}\n\
          endpoint = \[${registry_mirror_list}\]" \
        /etc/containerd/config.toml
    fi
    if [ $(sudo grep -c "${containerd_registry_mirror_gcr_marker}" /etc/containerd/config.toml) -eq 0 ]; then
      sudo sed -i "/${containerd_registry_mirror_marker}/a\\
        ${containerd_registry_mirror_gcr_marker}\n\
          endpoint = \[\"https:\/\/registry.aliyuncs.com\/google_containers\"\]" \
        /etc/containerd/config.toml
    fi
  fi

  # Setup cni plugin
  cni_plugin_url="https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz"
  cni_plugin_cache="${work_top_dir}/cni-plugins-linux-amd64-v1.3.0.tgz"
  sudo mkdir -p /opt/cni/bin || true
  if [ ! -e ${cni_plugin_cache} ]; then
    curl -SL ${cni_plugin_url} -o ${cni_plugin_cache}
  fi
  sudo tar -zxvf ${cni_plugin_cache} -C /opt/cni/bin

  # Setup network proxy
  docker_proxy_dir="/etc/systemd/system/docker.service.d"
  containerd_proxy_dir="/etc/systemd/system/containerd.service.d"
  sudo mkdir -p ${docker_proxy_dir} || true
  sudo mkdir -p ${containerd_proxy_dir} || true
  cat <<EOF | sudo tee ${docker_proxy_dir}/proxy.conf
[Service]
EOF
  cat <<EOF | sudo tee ${containerd_proxy_dir}/proxy.conf
[Service]
EOF
  if [ "X$HTTPS_PROXY" != "X" ]; then
    cat <<EOF | sudo tee -a ${docker_proxy_dir}/proxy.conf
Environment="HTTPS_PROXY=${HTTPS_PROXY}"
EOF
    cat <<EOF | sudo tee -a ${containerd_proxy_dir}/proxy.conf
Environment="HTTPS_PROXY=${HTTPS_PROXY}"
EOF
  elif [ "X$https_proxy" != "X" ]; then
    cat <<EOF | sudo tee -a ${docker_proxy_dir}/proxy.conf
Environment="HTTPS_PROXY=${https_proxy}"
EOF
    cat <<EOF | sudo tee -a ${containerd_proxy_dir}/proxy.conf
Environment="HTTPS_PROXY=${https_proxy}"
EOF
  fi
  if [ "X$HTTP_PROXY" != "X" ]; then
    cat <<EOF | sudo tee -a ${docker_proxy_dir}/proxy.conf
Environment="HTTP_PROXY=${HTTP_PROXY}"
EOF
    cat <<EOF | sudo tee -a ${containerd_proxy_dir}/proxy.conf
Environment="HTTP_PROXY=${HTTP_PROXY}"
EOF
  elif [ "X$http_proxy" != "X" ]; then
    cat <<EOF | sudo tee -a ${docker_proxy_dir}/proxy.conf
Environment="HTTP_PROXY=${http_proxy}"
EOF
    cat <<EOF | sudo tee -a ${containerd_proxy_dir}/proxy.conf
Environment="HTTP_PROXY=${http_proxy}"
EOF
  fi
  if [ "X$NO_PROXY" != "X" ]; then
    cat <<EOF | sudo tee -a ${docker_proxy_dir}/proxy.conf
Environment="NO_PROXY=${NO_PROXY}"
EOF
    cat <<EOF | sudo tee -a ${containerd_proxy_dir}/proxy.conf
Environment="NO_PROXY=${NO_PROXY}"
EOF
  elif [ "X$no_proxy" != "X" ]; then
    cat <<EOF | sudo tee -a ${docker_proxy_dir}/proxy.conf
Environment="NO_PROXY=${no_proxy}"
EOF
    cat <<EOF | sudo tee -a ${containerd_proxy_dir}/proxy.conf
Environment="NO_PROXY=${no_proxy}"
EOF
  fi

  sudo systemctl daemon-reload
  sudo systemctl restart containerd
  sudo systemctl restart docker
}

install_docker_containerd

install_k8s () {
  echo "Install k8s"
  # install kubeadm+kubelet+kubectl
  cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
exclude=kubelet kubeadm kubectl
EOF
  sudo yum install -y kubelet-1.28.0 kubeadm-1.28.0 kubectl-1.28.0 --disableexcludes=kubernetes
  sudo systemctl enable --now kubelet

  cat <<EOF | sudo tee /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS=--cgroup-driver=cgroupfs
EOF

  sudo systemctl daemon-reload
  sudo systemctl restart kubelet
}

install_k8s

load_k8s_images () {
  echo "Loading cached k8s images"
  pushd ${work_top_dir}
  # load k8s images from local FS can speedup deploy
  if [ -d k8s.init.images ]; then
    for f in $(ls k8s.init.images/*.tar); do
      sudo ctr -n k8s.io image import $f
    done
  fi
  popd # ${work_top_dir}
}

load_k8s_images

init_k8s_control_plane () {
  echo "Initiate k8s control-plane"

  if ! grep -qxF "127.0.0.1   ${k8s_node_name}" /etc/hosts; then
    echo "127.0.0.1   ${k8s_node_name}" | sudo tee -a /etc/hosts
  fi

  sudo -E kubeadm init --kubernetes-version=v1.28.0 \
    --node-name=${k8s_node_name} \
    --image-repository registry.aliyuncs.com/google_containers \
    --pod-network-cidr=10.244.0.0/16

  rm $HOME/.kube -rf || true
  mkdir $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

  # apply net cni (flannel)
  kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

  # setup roles
  if [ "X${k8s_node_role}" = "Xcontrol-plane+worker" ]; then
    kubectl taint nodes ${k8s_node_name} node-role.kubernetes.io/control-plane:NoSchedule-
    kubectl label nodes ${k8s_node_name} node.kubernetes.io/worker=
  fi
  kubectl get nodes --show-labels
}

case "${k8s_node_role}" in
"control-plane"|"control-plane+worker")
  init_k8s_control_plane
  ;;
*)
  ;;
esac

install_k8s_tools () {
  echo "Install k8s tools"
  kustomize_cache="${work_top_dir}/kustomize_v5.4.3_linux_amd64.tar.gz"
  if [ ! -e ${kustomize_cache} ]; then
    curl -SL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.4.3/kustomize_v5.4.3_linux_amd64.tar.gz \
      -o ${kustomize_cache}
  fi
  sudo tar -zxvf ${kustomize_cache} -C /usr/local/bin/
  sudo chmod +x /usr/local/bin/kustomize
}

install_k8s_tools
